package org.example;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * @author Myli
 * @create 2023-02-15 23:23
 */
public class Scrap {

    //规定要爬取的网页
    static String url = "https://www.skj520.com";
    static String url0 = "https://www.skj520.com/a/98521/98521123/index.html";

    public static void main(String[] args) throws IOException {

        ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(
                16,
                32,
                10,
                TimeUnit.SECONDS, new LinkedBlockingQueue<>());

        //设置下载文件存放磁盘的位置
        File path = new File("C:\\Users\\Xyulu\\Desktop\\高天之上");
        //判断文件夹是否存在，不存在就创建
        if (!path.exists()) {
            path.mkdirs();
        }
        Connection conn = Jsoup.connect(url0);
        //伪装成为浏览器，有的网站爬取数据会阻止访问，伪装成浏览器可以访问，这里我伪装成Google浏览器
        Document doc = conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36").get();
        System.out.println("开始连接网站……");
        //寻找名叫panel-body的div，接着再在里面寻找叫list-group.list-charts的ul，再在里面寻找li，再在li里寻找a    //"div.panel-body ul.list-group.list-charts li a"
        Elements as = doc.select(".listmain").select("dd");
        if (as.isEmpty()) {
            System.out.println("未查到任何内容，请检查标签是否正确！");
        } else {
            ArrayList<Element> elements = new ArrayList<>();
            //去除列表最近 12条非章节数据
            for (int a = 0; a < 12; a++) {
                elements.add(a, as.get(0));
                as.remove(0);
            }
            for (Element a : as) {
                //对于每一个元素取出拿到href和小说名字
//                String href = a.attr("href");
                String href = a.getElementsByAttribute("href").attr("href");
                String title = a.text().replace(" ", "")
                        .replace("?", "");
                //path:表示存放文件的位置
                //href:表示要读取内容的页面
                //title:表示存放文件名称
                if (path.exists() || !href.isEmpty() || !title.isEmpty()) {
                    poolExecutor.execute(() -> save(path, href, title));
//                    save(path, href, title);
                } else {
                    System.out.println("！！！！！！！！！！！！！！");
                }
            }
            poolExecutor.shutdown();
        }
    }

    private static void save(File path, String href, String title) {
        Writer out = null;
        String url2 = "";
        Connection conn = null;
        Document doc = null;
        String content = "";
        String path1 = "";
        int n = 0;
        File f = null;
        //在file目录下创建每一个章命名的txt文件
        path1 = path + "\\" + title + ".txt";
        f = new File(path1);
        try {
            //构建输出流对象，因为小说的内容是字符类型的数据
            if (!path.exists()) {
                // 判断路径是否存在
                if (!path.exists()) {
                    // 如果不存在则创建目录
                    path.mkdir();
                    f.createNewFile();
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        try {
            out = new FileWriter(f);
            //构建读取页面的url2
            url2 = url + href;
            //正确的地址为https://www.7722wx.com/html/69528/752585192.html
            conn = Jsoup.connect(url2);
            //伪装成为浏览器
            doc = conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36").get();
            //获取小说的正文
            content = doc.select("div[class=showtxt]div[id=content]").html();
            //处理特殊数据
            content = content.replace("<script>app2();</script>", "");
            content = content.replace("<script>chaptererror();</script>", "");
            content = content.replace("<br>", "");
            content = content.replace("&nbsp;", "");
            content = content.replace("　　", "\r\n  ");
            out.write(title + "\r\n" + content);
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (f.exists()) {
            System.out.println(title + "创建成功！");
        } else {
            System.out.println(title + "！！！！！！！！！");
        }
        //需要使用休眠，为了防止网站检测为蓄意攻击，停止我们的IP访问
//        n = (int) (Math.random() * 10);
//        try {
//            Thread.sleep(n);
//        } catch (InterruptedException e) {
//            // TODO Auto-generated catch block
//            e.printStackTrace();
//        }
    }
}
